#coding=utf-8
import json
import configparser
import os
#import GraphGenerator
import databaseSqlite3 as database
import transKafka as tkfk
from copy import deepcopy as dc


#ConfigParser用来读取配置文件

class InterleavedLogAnalysis:


    def __init__(self):
        self.cp = configparser.ConfigParser()
        self.cp.read(os.path.split(os.path.realpath(__file__))[0] + '/Config')
        #file1和file2需要一一对应，即一个日志模版对应一个原始日志内容！！！
        #self.file1 = open(self.cp.get('FilePath','templateFile'),'r')    #日志模版文件
        self.file2 = open(self.cp.get('FilePath','originalLogFile'),'r') #JSON格式日志原文件
        #self.timeSequence = [] #每个日志的时间序列
        #self.logTypeSequence = [] #每个日志的类型序列
        self.nodesInGraph = [] #转移图中的节点信息
        self.transition = [] # 转移图
        self.distance = []  #状态之间的转移距离
        self.seeds = []  #种子日志模版集合
        self.seedsQueue = [] #种子日志模版队列
        self.visited = []
        self.trace = []
        self.has_circle = False
        self.circles = []


    #取出日志原文件每一条日志的json格式字符串,并装载为一个list
    # def logPreProcess(self):
    #     jsonlist = []
    #     for line in self.file2:
    #         jsonlist.append(json.loads(line, strict=False))#json.loads得到dict
    #     return jsonlist
    def logPreProcess(self):
        kfktranse = tkfk.Kafka_Transe()
        jsonlist = kfktranse.consumer_demo()
        return jsonlist




    #提取原日志时间戳序列
    def timeSequenceGen(self,jsonlist,logTypeSequence):
        timeStamp = []

        for i in range(len(jsonlist)):
            if not jsonlist[i].__contains__('Time'):#jsonlist是list<dict<k,v>>形式
                logTypeSequence[i] = '0'
                timeStamp.append(0)
            else:
                timeStamp.append(int(jsonlist[i]['Time']))
        return timeStamp


    #提取日志类型序列
    def logTypeSequenceGen(self,jsonlist):
        logTypeSequence = []
        for i in range(len(jsonlist)):
            logTypeSequence.append(int(jsonlist[i]['EventId']))
        return logTypeSequence
    # def logTypeSequenceGen(self):
    #     sequence = self.file1.read()
    #     logTypeSequence = json.loads(sequence)['id']
    #     #loads,字符串化成字典dict
    #     return logTypeSequence


    #计算相邻两个日志之间的时间差(距离)，postIndex是后一个日志下标
    def distanceCompute(self,postIndex,timeSequence):
        pre_node = timeSequence[postIndex-1]
        post_node = timeSequence[postIndex]
        distance = post_node - pre_node
        return distance

    #计算任意两点（日志）之间的距离（时间间隔）
    def distanceBetweenTwoNode(self,timeSequence,x,y):
        distance = timeSequence[y] - timeSequence[x]
        return distance

    #建立时间序列模型
    def modelGen(self,logTypeSequence,timeSequence,jump,startnode,lim):#模板类型序列，时间戳序列
        models = []
        nodesInWindow = []
        supportForNodesInWindow = []
        for i in range(len(timeSequence)):
            if logTypeSequence[i] == startnode:
            #找到startnode这个点
                model = []
                num = 1 #代表第几个时间窗口
                t = i + 1
                while t < len(timeSequence):
                    if num > lim:#num<=10
                        break
                    distance = self.distanceBetweenTwoNode(timeSequence, i, t)
                    #print 'dist:%d' % (distance)
                    # 如果当前日志在当前时间窗口内
                    if jump * (num-1) <= distance < jump * num:
                        if logTypeSequence[t] in nodesInWindow:#模板类型已经在nodesInWindow里
                            t += 1
                            continue
                        else:
                            #如果该时间窗口内没有该类型的日志则加入该类型的日志
                            nodesInWindow.append(logTypeSequence[t])
                            supportForNodesInWindow.append(1)#?干啥的
                        t += 1

                    else:
                    #如果当前日志超出了时间窗口
                        num += 1
                        timeWindow = []
                        timeWindow.append(nodesInWindow)
                        timeWindow.append(supportForNodesInWindow)
                        model.append(timeWindow)
                        nodesInWindow = []#清空这俩
                        supportForNodesInWindow = []
                        #t -= 1#下一轮还看这个t
                #print model
                models.append(model)
        return models

    #模型融合,生成二级模型，把这个类型的所有日志的窗口叠加
    def modelCombination(self,models):
        if len(models) == 0:
            return []
        finalModel = models[0]#model
        for window in finalModel:#timewindow
            for i in range(len(window[1])):
                window[1][i] -= 1
                #supportForNodesInWindow -1

        for model in models:
            for i in range(len(model)):#i--第i窗口
                for logtype in model[i][0]:#这个窗口里的日志类型type
                    if logtype in finalModel[i][0]:
                        finalModel[i][1][finalModel[i][0].index(logtype)] += 1
                    else:
                        finalModel[i][0].append(logtype)
                        finalModel[i][1].append(1)
        return finalModel

    #根据二级模型生成支撑度，
    def supportGen(self,finalModel,support,sum):
        if len(finalModel) == 0:
            return []
        result = []
        thisstart = []
        thisend = []
        thislogSequence = []
        for i in range(len(finalModel)):
            for j in range(len(finalModel[i][0])):
                if finalModel[i][1][j] >= support * sum:#i窗口后继模板j类型数量大于阈值
                    thisstart.append(i)#i个窗口
                    thisend.append(i)
                    thislogSequence.append(finalModel[i][0][j])
                    limit = 1
                    for k in range(i+1,len(finalModel)):
                        if limit > 1 or k >= len(finalModel):
                            break
                        if finalModel[i][0][j] in finalModel[k][0]:
                        #若#k窗口后继模板含j
                            thisTemplateIndex = finalModel[k][0].index(finalModel[i][0][j])
                            if finalModel[k][1][thisTemplateIndex] >= support * sum:
                            #k窗口后继模板j类型数量大于阈值
                                thisend[len(thisend)-1] = k
                                limit = 1
                            else:
                                limit += 1
                        else:
                            limit += 1

                        #break
        # print thisstart
        # print thisend
        # print thislogSequence
        result.append(thislogSequence)
        result.append(thisstart)
        result.append(thisend)
        return result

    #根据timestamp升序排列
    def naiveSort(self,timestamp,logtypesequence,logjson):
        for i in range(1, len(timestamp)):
            for j in range(0,len(timestamp) - i):
                if timestamp[j]>timestamp[j + 1]:
                    timestamp[j], timestamp[j+1]= timestamp[j+1], timestamp[j]
                    logtypesequence[j], logtypesequence[j+1]= logtypesequence[j+1], logtypesequence[j]
                    logjson[j], logjson[j+1]= logjson[j+1], logjson[j]
        #print timestamp
        result = []
        result.append(timestamp)
        result.append(logtypesequence)
        result.append(logjson)
        return result

    #原生成FSA入口（包含整个流程）
    '''
    def FSAGen(self,logTypeSequence,timeSequence,jump,seed):
        #self.seeds.append(seed)
        models = self.modelGen(logTypeSequence,timeSequence,jump,seed)
        finalmodels = self.modelCombination(models)
        result = self.supportGen(finalmodels,0.5,len(models))
        print result
        #当有一个新节点的时候,修改transition矩阵和distance矩阵
        if not seed in self.nodesInGraph:
            list1 = []
            list2 = []
            print '加入元素:' + ' ' + str(seed)
            self.nodesInGraph.append(seed)
            for i in range(len(self.transition)):
                self.transition[i].append(0)
                self.distance[i].append([0,0])
            for node in self.nodesInGraph:
                list1.append(0)
                list2.append([0,0])
            self.transition.append(list1)
            self.distance.append(list2)
        if len(result) == 0:
            if len(self.seedsQueue) > 0:
                self.FSAGen(logTypeSequence,timeSequence,jump,self.seedsQueue.pop(0))
                return
            else:
                return

        indexOfSeed = self.nodesInGraph.index(seed)
        #查找seed的前一个节点
        pre_seed = []
        for i in range(len(self.nodesInGraph)):
                    if (self.transition[i][indexOfSeed]) == 1 and (indexOfSeed != i):
                        pre_seed.append(i)
        #初始化更新矩阵信息
        print '---------------初始化更新矩阵信息----------------'
        for i in range(len(result[0])):
            #如果没有生成模型则将该点加入队列,作为以后的seed
            if not result[0][i] in self.seeds:
                self.seeds.append(result[0][i])
                self.seedsQueue.append(result[0][i])
            if result[0][i] in self.nodesInGraph:
                index = self.nodesInGraph.index(result[0][i])
                if self.transition[indexOfSeed][index] == 1:
                    continue
                else:
                    print '建立链接:' + str(self.nodesInGraph[indexOfSeed]) + ' ' + str(self.nodesInGraph[index])
                    self.transition[indexOfSeed][index] = 1
                    self.distance[indexOfSeed][index][0] = result[1][i]
                    self.distance[indexOfSeed][index][1] = result[2][i]

            else:

                list1 = []
                list2 = []
                print '加入元素:' + ' ' + str(result[0][i])
                self.nodesInGraph.append(result[0][i])
                for i in range(len(self.transition)):
                    self.transition[i].append(0)
                    self.distance[i].append([0,0])
                for node in self.nodesInGraph:
                    list1.append(0)
                    list2.append([0,0])
                self.transition.append(list1)
                self.distance.append(list2)
                index = len(self.nodesInGraph)-1
                self.transition[indexOfSeed][index] = 1
                print '建立链接:' + str(self.nodesInGraph[indexOfSeed]) + ' ' + str(self.nodesInGraph[index])
                self.distance[indexOfSeed][index][0] = result[1][i]
                self.distance[indexOfSeed][index][1] = result[2][i]

        #合并矩阵
        print '合并矩阵....'
        for i in range(len(result[0])):
            index = self.nodesInGraph.index(result[0][i])
            if index == indexOfSeed:
                continue
            for preseed in pre_seed:
                if preseed == indexOfSeed:
                    continue
                if self.transition[preseed][index] == 1:
                    if self.transition[index][indexOfSeed] == 1:
                        continue

                    start_twonodes = self.distance[preseed][indexOfSeed][0] + self.distance[indexOfSeed][index][0]
                    end_twonodes = self.distance[preseed][indexOfSeed][1] + self.distance[indexOfSeed][index][1]
                    start_onenode = self.distance[preseed][index][0]
                    end_onenode = self.distance[preseed][index][1]
                    if start_twonodes <= start_onenode and end_twonodes >= end_onenode:


                        print '删除链接:' + str(self.nodesInGraph[preseed]) + ' ' + str(self.nodesInGraph[index])
                        self.transition[preseed][index] = 0
                        self.distance[preseed][index][0] = 0
                        self.distance[preseed][index][1] = 0
        if len(self.seedsQueue) == 0:
            for i in range(len(result[0]))[::-1]:
                lastIndex =  self.nodesInGraph.index(result[0][i])
                for j in range(i):
                    midIndex = self.nodesInGraph.index(result[0][j])
                    if indexOfSeed == midIndex or midIndex == lastIndex:
                        continue

                    if self.transition[midIndex][lastIndex] == 1:
                        if self.transition[lastIndex][midIndex] == 1:
                            continue

                        start_twonodes = self.distance[indexOfSeed][midIndex][0] + self.distance[midIndex][lastIndex][0]
                        end_twonodes = self.distance[indexOfSeed][midIndex][1] + self.distance[midIndex][lastIndex][1]
                        start_onenode = self.distance[indexOfSeed][lastIndex][0]
                        end_onenode = self.distance[indexOfSeed][lastIndex][1]
                        if start_twonodes <= start_onenode and end_twonodes >= end_onenode:

                            print '删除链接:' + str(self.nodesInGraph[indexOfSeed]) + ' ' + str(self.nodesInGraph[lastIndex])
                            self.transition[indexOfSeed][lastIndex] = 0
                            self.distance[indexOfSeed][lastIndex][0] = 0
                            self.distance[indexOfSeed][lastIndex][1] = 0
                            break

        if len(self.seedsQueue) > 0:
            self.FSAGen(logTypeSequence,timeSequence,jump,self.seedsQueue.pop(0))
        '''
    #返回post这个模板类型在finalModel的i个窗口中，数量最多的那个窗口index
    def getTimeWindowOfMaxAPNum(self,finalModel,post):
        maxTimeWindow = 0
        for i in range(len(finalModel)):
                if post in finalModel[i][0]:
                    if post not in finalModel[0][0] and maxTimeWindow == 0:
                        maxTimeWindow = i
                    if finalModel[i][1][finalModel[i][0].index(post)] >= finalModel[maxTimeWindow][1][finalModel[i][0].index(post)]:
                        maxTimeWindow = i

        #print "maxTimeWindow" + str(maxTimeWindow)
        return maxTimeWindow

    #由seed的加入，更新transition和distance矩阵
    def MatrixReverse(self,logTypeSequence,timeSequence,jump,seed,support,lim):
        #self.seeds.append(seed)
        models = self.modelGen(logTypeSequence,timeSequence,jump,seed,lim)
        finalmodels = self.modelCombination(models)
        result = self.supportGen(finalmodels,support,len(models))

        #当有一个新节点的时候,修改transition矩阵和distance矩阵
        if not seed in self.nodesInGraph:
        ##seed不在转移图中的节点信息
            list1 = []
            list2 = []
            print('add element :' + ' ' + str(seed))
            self.nodesInGraph.append(seed)
            # 两个矩阵扩容
            for i in range(len(self.transition)):
                self.transition[i].append(0)
                self.distance[i].append([0,0])
            for node in self.nodesInGraph:
                list1.append(0)
                list2.append([0,0])
            self.transition.append(list1)
            self.distance.append(list2)
        #
        if len(result) == 0:
            if len(self.seedsQueue) > 0:
                self.MatrixReverse(logTypeSequence,timeSequence,jump,self.seedsQueue.pop(0),support,lim)
                return
            else:
                return

        indexOfSeed = self.nodesInGraph.index(seed)
        #查找seed的前一个节点下标（图中）
        pre_seed = []
        for i in range(len(self.nodesInGraph)):
                    if (self.transition[i][indexOfSeed]) == 1 and (indexOfSeed != i):
                        pre_seed.append(i)
        #初始化更新矩阵信息
        print('---------------初始化更新矩阵信息----------------')
        for i in range(len(result[0])):
            #如果没有生成模型则将该点加入队列,作为以后的seed
            if not result[0][i] in self.seeds:
                self.seeds.append(result[0][i])
                self.seedsQueue.append(result[0][i])
            if result[0][i] in self.nodesInGraph:
                # seed和它大于阈值的后继模板建立连接
                index = self.nodesInGraph.index(result[0][i])
                if self.transition[indexOfSeed][index] == 1:
                    continue
                else:
                    print('connect :' + str(self.nodesInGraph[indexOfSeed]) + ' ' + str(self.nodesInGraph[index]))
                    self.transition[indexOfSeed][index] = 1
                    self.distance[indexOfSeed][index][0] = result[1][i]#起始窗口
                    self.distance[indexOfSeed][index][1] = result[2][i]#大于阈值的结束窗口
            else:
            #此模板不在nodesInGraph中
                list1 = []
                list2 = []
                print('add element :' + ' ' + str(result[0][i]))
                self.nodesInGraph.append(result[0][i])
                for j in range(len(self.transition)):
                    self.transition[j].append(0)
                    self.distance[j].append([0,0])
                for node in self.nodesInGraph:
                    list1.append(0)
                    list2.append([0,0])
                self.transition.append(list1)
                self.distance.append(list2)
                index = len(self.nodesInGraph)-1
                self.transition[indexOfSeed][index] = 1
                print('connect :' + str(self.nodesInGraph[indexOfSeed]) + ' ' + str(self.nodesInGraph[index]))
                self.distance[indexOfSeed][index][0] = result[1][i]
                self.distance[indexOfSeed][index][1] = result[2][i]
            if len(self.seedsQueue) > 0:
                self.MatrixReverse(logTypeSequence,timeSequence,jump,self.seedsQueue.pop(0),support,lim)

    #更新矩阵 i->j->k 和i->k 看是线性还是两条路
    def MatrixClean(self,logTypeSequence,timeSequence,jump):
        tempMatrix = self.transition
        for i in range(len(self.nodesInGraph)):
            models = self.modelGen(logTypeSequence,timeSequence,jump,self.nodesInGraph[i],lim)
            #模板nodesInGraph[i]的models
            finalmodels = self.modelCombination(models)
            for j in range(len(self.nodesInGraph)):
                if i == j or self.transition[i][j] == 0:
                    continue

                for k in range(len(self.nodesInGraph)):
                    if i == k or j == k:
                        continue
                    if self.transition[j][k] == 1 and self.transition[i][k] == 1:
                        AB = self.getTimeWindowOfMaxAPNum(finalmodels,self.nodesInGraph[j])
                        #nodesInGraph[j]在finalmodels i个窗口中，出现次数最多的那个窗口
                        AC = self.getTimeWindowOfMaxAPNum(finalmodels,self.nodesInGraph[k])
                        #print(str(AB) + ' ' + str(AC))
                        if AB < AC:
                        #说明T2紧跟T1出现的多
                            print('delete connect : ' + str(self.nodesInGraph[i]) + ' ' + str(self.nodesInGraph[k]))
                            tempMatrix[i][k] = 0
        self.transition = tempMatrix

    def dfs(self, node_index):
    #找到图中所有的环
        if (node_index in self.visited):
            if (node_index in self.trace):
                has_circle = True
                trace_index = self.trace.index(node_index)
                tCircle = []
                for i in range(trace_index, len(self.trace)):
                    tCircle.append(i)
                    #print(str(self.trace[i]) + ' ', end='')
                self.circles.append(tCircle)
                #print('\n', end='')
                return
            return

        self.visited.append(node_index)
        self.trace.append(node_index)

        # if (node_index != ''):
        #     children = child_graph[node_index].split('#')
        #     for child in children:
        #         dfs(child)
        for i in range(len(self.transition[node_index])):
            if (self.transition[node_index][i] == 1):
                self.dfs(i)
        self.trace.pop()


if __name__ == "__main__":
    interleave = InterleavedLogAnalysis()
    jsonlist = interleave.logPreProcess()
    logSequence = interleave.logTypeSequenceGen(jsonlist)
    timestamp = interleave.timeSequenceGen(jsonlist,logSequence)

    #时间标签序列
    timeStamp_ready = []
    #json序列(日志全部信息序列)
    logJson_ready = []
    #日志模版序列
    logTypeSequence_ready = []

    for i in range(len(timestamp))[::]:
        if timestamp[i] != 0:
            timeStamp_ready.append(timestamp[i])
            logJson_ready.append(jsonlist[i])
            logTypeSequence_ready.append(logSequence[i])
    result = interleave.naiveSort(timeStamp_ready,logTypeSequence_ready,logJson_ready)

    #训练模型
    #?参数什么意思
    jump = float(interleave.cp.get('Params','jump'))
    startnode = int(interleave.cp.get('Params','startnode'))
    lim = float(interleave.cp.get('Params','lim'))
    support = float(interleave.cp.get('Params','support'))
    interleave.MatrixReverse(result[1],result[0],jump,startnode,support,lim)
    interleave.MatrixClean(result[1],result[0],0.1)
    interleave.dfs(0)
    print("Circles : ")
    for i in range(len(interleave.circles)):
        for j in range(len(interleave.circles[i])):
            print(str(interleave.circles[i][j]), end=" ")
        print("")

    #处理环路

    #输出邻接矩阵
    for i in range(len(interleave.transition)):
        for j in range(len(interleave.transition[i])):
            print(str(interleave.transition[i][j]), end = " ")
        print("")
    #存入数据库
    db = database.databaseSql()
    graphID = db.insert(interleave.transition, interleave.distance)
    print("GraphId = " + str(graphID))
    # 模型作图
    # graphGen = GraphGenerator.drawGraph()
    # graphGen.drawGraphviz(interleave.nodesInGraph,interleave.transition)
    #持久化模型
    #file1 = open('E:\Py\CFGMining_New\data\FSA_002.result', 'w')
    #FSA = ''
    #for x in range(len(interleave.nodesInGraph)):
        #print interleave.nodesInGraph[x]
        #FSA = FSA + str(interleave.nodesInGraph[x]) + ' '
        #for y in range(len(interleave.nodesInGraph)):
            #if interleave.transition[x][y] == 1:
                #FSA = FSA + str(interleave.nodesInGraph[y]) + ' '
        #FSA = FSA + '\n'
    #file1.write(FSA)

